﻿/**
* @author alex
*/
Namespace("sketch.framework.text");
(function () {
    var txt = sketch.framework.text;

    function trim(str){
        return str.replace(/^\s+|\s+$/g, '');
    }

    txt.Tokenizer = {};
    txt.Tokenizer.tokenizeLine = function(/*str*/str){
        var start = 0;
        var i = 0, length=str.length;
        var tokens = [];
              
        for (; i < length; i++) {
            if(str[i] === " "){
                // skip spaces
                for (; i < length && str[i] === " "; i++) {}

                var token = {
                    value : str.substr(start, i - start)
                };
                token.length = token.value.length;
                token.value = trim(token.value);

                tokens.push(token);
                start = i;
            }
        }

        if(start !== length){
            var token = {
                value:str.substr(start, i - start)
            };
            token.length = token.value.length;
            token.value = trim(token.value);
            tokens.push(token);
        }

        return tokens;
    };

    txt.Tokenizer.tokenize = function(str){
        str = str.replace("\r", "");
        var lines = str.split("\n");
        var tokenLines = [];
        var tokenizeLine = txt.Tokenizer.tokenizeLine;
        for (var i = 0, length = lines.length; i < length; i++) {
            var tokens = tokenizeLine(lines[i]);
            tokenLines.push(tokens);
        }

        return tokenLines;
    }
})();
